{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = \"\"\"\n",
    "# translation/malay-english/translated-0.json\n",
    "# translation/malay-english/translated-100000.json\n",
    "# translation/malay-english/translated-200000.json\n",
    "# translation/malay-english/translated-300000.json\n",
    "# translation/malay-english/translated-400000.json\n",
    "# translation/malay-english/translated-500000.json\n",
    "# translation/malay-english/translated-600000.json\n",
    "# translation/malay-english/translated-700000.json\n",
    "# translation/malay-english/translated-800000.json\n",
    "# translation/malay-english/translated-900000.json\n",
    "# translation/malay-english/translated-1000000.json\n",
    "# translation/malay-english/translated-1100000.json\n",
    "# translation/malay-english/translated-1200000.json\n",
    "# translation/malay-english/translated-1300000.json\n",
    "# translation/malay-english/translated-1400000.json\n",
    "# translation/malay-english/translated-1500000.json\n",
    "# translation/malay-english/translated-1600000.json\n",
    "# translation/malay-english/translated-1700000.json\n",
    "# translation/malay-english/translated-1800000.json\n",
    "# translation/malay-english/translated-1900000.json\n",
    "# translation/malay-english/translated-2000000.json\n",
    "# translation/malay-english/translated-2100000.json\n",
    "# translation/malay-english/translated-2200000.json\n",
    "# translation/malay-english/translated-2300000.json\n",
    "# translation/malay-english/translated-2400000.json\n",
    "# translation/malay-english/translated-2500000.json\n",
    "# translation/malay-english/translated-2600000.json\n",
    "# translation/malay-english/translated-2700000.json\n",
    "# translation/malay-english/translated-2800000.json\n",
    "# translation/malay-english/translated-2900000.json\n",
    "# translation/malay-english/translated-3000000.json\n",
    "# translation/malay-english/translated-3100000.json\n",
    "# translation/malay-english/translated-3200000.json\n",
    "# translation/malay-english/translated-3300000.json\n",
    "# translation/malay-english/translated-3400000.json\n",
    "# translation/malay-english/translated-3500000.json\n",
    "# \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# url = 'https://f000.backblazeb2.com/file/malay-dataset/'\n",
    "\n",
    "# for row in x.split('\\n'):\n",
    "#     if not len(row):\n",
    "#         continue\n",
    "#     link = f'{url}{row}'\n",
    "#     os.system(f'wget {link}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/parliament/translated-trainset-parliament.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/gnome-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/kde4-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/opensubtitle-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/qed-ms-en.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/tanzil-ms-en.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/ubuntu-ms-en.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('translated-trainset-parliament.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected = ['PERTANYAAN-PERTANYAAN JAWAB LISAN', 'PENGGAL KEEMPAT', 'PUSAT JAGAAN BERDAFTAR',\n",
    "           'BILANGAN PUSAT JAGAAN', 'pewan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50460\n"
     ]
    }
   ],
   "source": [
    "selected, reject = [], []\n",
    "for row in data:\n",
    "    if any([r.lower() in row[0].lower() for r in rejected]):\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    s = row[0]\n",
    "    if (sum(c.isdigit() for c in s) / len(s)) > 0.15:\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    if sum(c.isalpha() for c in s) == 0:\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    selected.append(row)\n",
    "    \n",
    "print(len(selected))\n",
    "x_parliament, y_parliament = list(zip(*selected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "with open('ubuntu-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "X, Y = list(zip(*data))\n",
    "X = list(X)\n",
    "Y = list(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('qed-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "x, y = list(zip(*data))\n",
    "X.extend(x)\n",
    "Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tanzil-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "x, y = list(zip(*data))\n",
    "X.extend(x)\n",
    "Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['translated-3200000.json',\n",
       " 'translated-700000.json',\n",
       " 'translated-2100000.json',\n",
       " 'translated-3300000.json',\n",
       " 'translated-2300000.json',\n",
       " 'translated-2000000.json',\n",
       " 'translated-600000.json',\n",
       " 'translated-900000.json',\n",
       " 'translated-1000000.json',\n",
       " 'translated-1100000.json',\n",
       " 'translated-1900000.json',\n",
       " 'translated-500000.json',\n",
       " 'translated-1400000.json',\n",
       " 'translated-1500000.json',\n",
       " 'translated-2600000.json',\n",
       " 'translated-200000.json',\n",
       " 'translated-2900000.json',\n",
       " 'translated-3400000.json',\n",
       " 'translated-3500000.json',\n",
       " 'translated-2800000.json',\n",
       " 'translated-300000.json',\n",
       " 'translated-2500000.json',\n",
       " 'translated-3100000.json',\n",
       " 'translated-1300000.json',\n",
       " 'translated-2400000.json',\n",
       " 'translated-100000.json',\n",
       " 'translated-1600000.json',\n",
       " 'translated-2700000.json',\n",
       " 'translated-0.json',\n",
       " 'translated-800000.json',\n",
       " 'translated-1800000.json',\n",
       " 'translated-2200000.json',\n",
       " 'translated-1200000.json',\n",
       " 'translated-1700000.json',\n",
       " 'translated-3000000.json',\n",
       " 'translated-400000.json']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "translated = glob('translated*0.json')\n",
    "translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in translated:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "\n",
    "    x, y = list(zip(*data))\n",
    "    X.extend(x)\n",
    "    Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3808696, 3808696)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3808696/3808696 [01:25<00:00, 44497.25it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from unidecode import unidecode\n",
    "import re\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string).replace('\\n', ' ').replace('\\t', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string\n",
    "\n",
    "filtered_X, filtered_Y = [], []\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    X[i] = cleaning(X[i])\n",
    "    Y[i] = cleaning(Y[i])\n",
    "    if len(X[i]) and len(Y[i]):\n",
    "        filtered_X.append(X[i])\n",
    "        filtered_Y.append(Y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3808696, 3808565)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(filtered_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Beliau pernah disenaraihitamkan sekitar 1950-an setelah beliau menyertai \"House Un-American Activities Committee\" yang juga menjadi Parti Komunis.',\n",
       "  'He was blacklisted around the 1950s after joining the House Un-American Activities Committee, also a Communist Party.'),\n",
       " ('Kemudian beliau mencuba pula membawa peranan menerusi \"Paper Dolls\" (1984) dan \"Capital News\" (1990), yang diterbitkan oleh American Broadcasting Company (ABC), dan sekali lagi menerusi \"Harts of the West\" (1993), untuk CBS.',\n",
       "  'He then tried to lead roles in \"Paper Dolls\" (1984) and \"Capital News\" (1990), published by the American Broadcasting Company (ABC), and again through \"Harts of the West\" (1993), for CBS.'),\n",
       " ('Beliau pernah berlakon dalam \"The Loner\" (yang berakhir dengan satu musim sahaja pada 1965 hingga 1966), dua rancangan TV yang gagal menerusi stesyen NBC iaitu \"San Francisco International Airport\" (1970/71) dan \"Joe Forrester\" (1975/76).',\n",
       "  'He starred in \"The Loner\" (which ended with only one season between 1965 and 1966), two failed TV shows through the NBC stations \"San Francisco International Airport\" (1970/71) and \"Joe Forrester\" (1975/76) ).'),\n",
       " ('Beliau memulakan kerjaya beliau semula apabila nama beliau dipulihkan oleh anggota FBI dan mula menumpukan kepada pembikinan drama TV.',\n",
       "  'He started his career again when his name was restored by FBI agents and began focusing on making TV dramas.'),\n",
       " ('Beliau seterusnya membintangi rancangan antologi eponim \"The Lloyd Bridges Show\", yang turut menampilkan kedua-dua orang anak lelakinya Beau Bridges dan Jeff Bridges.',\n",
       "  'She then starred in the anthology show \"The Lloyd Bridges Show,\" which also featured her two sons Beau Bridges and Jeff Bridges.'),\n",
       " ('dan Harriet Evelyn Brown.', 'and Harriet Evelyn Brown.'),\n",
       " ('Beliau pernah menuntut di Sekolah Tinggi (Senior) Eureka pada 1931.',\n",
       "  'He studied at Eureka High School in 1931.'),\n",
       " ('Bridges mula dikenali setelah memegang watak sebagai Mike Nelson, menerusi siri TV \"Sea Hunt\", (1958 - 1961).',\n",
       "  'Bridges first became known after playing the role of Mike Nelson, in the TV series \"Sea Hunt\", (1958 - 1961).'),\n",
       " ('Beliau juga pernah ditawarkan watak sebagai \"Captain Kirk\" sebelum watak tersebut berpindah kepada William Shatner.',\n",
       "  'He was also offered the character as \"Captain Kirk\" before the character was transferred to William Shatner.'),\n",
       " ('Di Pantai St. Pete.', 'At St. Pete.')]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(filtered_X[-10:], filtered_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(filtered_Y, filtered_X, test_size = 100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('3 times 2 is 6, so our new numerator is 6.',\n",
       "  '3x2 sama dengan 6, jadi pengangka barunya adalah 6.'),\n",
       " ('HTML Entity Set: The first version of HTML is built into this small subset, related to the hosts available in three 8-bit Western fonts.',\n",
       "  'Set Entiti HTML: Versi awal HTML dibina dalam subset kecil ini, yang berkaitan dengan askara-askara yang terdapat dalam tiga fon Western 8-bit.'),\n",
       " ('The state religious council will present a fatwa to the Sultan, King or Yang di-Pertuan Agong for approval and subsequently to publish the fatwa.',\n",
       "  'Majlis agama negeri pula akan mempersembahkan fatwa kepada Sultan, Raja atau Yang di-Pertuan Agong untuk diperkenankan dan seterusnya untuk mewartakan fatwa berkenaan.'),\n",
       " ('\"We are not just trying to resolve the case but the steps to prevent it from happening again,\" he said.',\n",
       "  '\"Kita bukan setakat mahu selesaikan kes yang ada tetapi langkah untuk mencegahnya daripada berulang perlu diambil,\" katanya.'),\n",
       " ('Are they requited aught save what they used to do?',\n",
       "  'Mereka tidak diberikan balasan (pada hari akhirat) melainkan (bagi) apa yang mereka telah kerjakan (dari perbuatan kufur dan maksiat).'),\n",
       " ('3 \"gu\" in Seongnam (seongnam; Cheng Nan).',\n",
       "  '3 \"gu\" di Seongnam (seongnam; Cheng Nan ).'),\n",
       " ('Many of these authors consider the indigenous peoples of their day and culture comparable to those of the early humans and their culture.',\n",
       "  'Banyak penulis tersebut menganggap bahawa orang-orang asli sezaman dan budaya mereka sebanding dengan manusia-manusia awal dan budaya mereka.'),\n",
       " (\"In the meantime, Dr Izani said the absence of official documents that could prove the ability to repay housing loans was the main cause of the state's application for dismissal by the bank.\",\n",
       "  'Dalam pada itu, Dr Izani berkata, ketiadaan dokumen rasmi yang boleh membuktikan keupayaan membayar pinjaman perumahan menjadi punca utama permohonan penduduk negeri ini ditolak pihak bank.'),\n",
       " (\"She is also the husband of Malaysia's No. 1 singer, Datuk Siti Nurhaliza.\",\n",
       "  'Beliau juga adalah suami kepada penyanyi No 1 Malaysia, Datuk Siti Nurhaliza.'),\n",
       " ('when they come close to it, their ears, eyes and skins will testify against them for their misdeeds.',\n",
       "  'Sehingga apabila mereka sampai ke neraka, (maka) pendengaran dan penglihatan serta kulit-kulit badan mereka menjadi saksi terhadap mereka, mengenai apa yang mereka telah kerjakan.')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(train_X[-10:], train_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('subrufa \"was once recorded capturing and rescuing its larger prey like a dove;',\n",
       "  'subrufa\" pernah direkodkan menangkap dan melemaskan mangsa yang lebih besar daripada mereka seperti burung merpati; Dari atas permukaan air, perlakuan mereka sering disalahanggap seperti buaya yang sedang menerkam mangsa.'),\n",
       " ('Something needs to be done immediately by the Badminton Association of Malaysia (BAM) to prevent early losers from continuing, especially ahead of the Olympic qualifying campaign which begins next month.',\n",
       "  'Sesuatu perlu dilakukan segera oleh Persatuan Badminton Malaysia (BAM) untuk menghalang trend pemain kalah awal daripada terus berlaku, terutama menjelang kempen kelayakan ke Olimpik yang bermula bulan depan.'),\n",
       " (\"So she's going to make some distance up so he's not going to have to travel quite as far.\",\n",
       "  'Maka dia akan menghasilkan jarak'),\n",
       " ('On the contrary, this situation is quite different where forest growth has increased over the years.',\n",
       "  'Sebaliknya, situasi ini agak berbeza di mana pertumbuhan hutan telah meningkat selama beberapa tahun.'),\n",
       " ('[symbols have a _ prefix]', '[simbol mempunyai _ prefix]'),\n",
       " ('In July, the same year, the Selangor Zakat Center was pleased to donate the roofing provision for the Science Square, which was used as a prayer place for the boys during the evening session due to the existing surau jam.',\n",
       "  'Pada bulan Julai, tahun yang sama, Pusat Zakat Selangor telah berbesar hati menyumbangkan peruntukan pembinaan bumbung untuk Dataran Ilmu yang digunakan sebagai tempat solat murid-murid lelaki pada sesi petang kerana kesesakan surau sedia ada.'),\n",
       " (\"In addition to the exciting action on the field, football fans are also being treated to intense competition off the field, especially by the Kelantan Football Association (KAFA) 's endless story that will hold an Extraordinary General Meeting (EGM) on July 6.\",\n",
       "  'Selain aksi menarik di atas padang, peminat bola sepak turut disajikan saingan hangat di luar padang terutamanya kisah tidak berkesudahan Persatuan Bola Sepak Kelantan (KAFA) yang akan mengadakan Mesyuarat Agung Luar Biasa (EGM) pada 6 Julai depan.'),\n",
       " ('Its minister, Zuraida Kamaruddin, said the rebranding was not just in the name, but also included images, business service ethics and the process of obtaining a business license as a lender.',\n",
       "  'Menterinya, Zuraida Kamaruddin, berkata penjenamaan semula itu bukan sahaja dari segi nama, malah merangkumi imej, etika perkhidmatan perniagaan dan proses untuk mendapatkan lesen perniagaan sebagai peminjam wang.'),\n",
       " (\"Penta Security's first IoT security solution was launched in 2015.\",\n",
       "  'Solusi sekuriti IoT Penta Security yang pertama dilancarkan pada tahun 2015.'),\n",
       " ('According to the Indian census in 2001, Mahrajganj had a population of 6,027.',\n",
       "  'Menurut bancian India pada tahun 2001, Mahrajganj memiliki bilangan penduduk seramai 6,027.')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(test_X[-10:], test_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "train_X, train_Y = shuffle(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-en-to-ms.json', 'w') as fopen:\n",
    "    json.dump({'train_X': train_X, 'train_Y': train_Y, 'test_X': test_X, 'test_Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir train-en\n",
    "!mkdir test-en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train-en/left.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_X))\n",
    "    \n",
    "with open('train-en/right.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_Y))\n",
    "    \n",
    "with open('test-en/left.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(test_X))\n",
    "    \n",
    "with open('test-en/right.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(test_Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train-en/\n",
      "train-en/left.txt\n",
      "train-en/right.txt\n",
      "test-en/\n",
      "test-en/left.txt\n",
      "test-en/right.txt\n"
     ]
    }
   ],
   "source": [
    "!tar -czvf train-en-ms.tar.gz train-en\n",
    "!tar -czvf test-en-ms.tar.gz test-en"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
